import psycopg2
import time
import math
import numpy as np
import os.path
import pandas as pd
import re
import pycld2
import nltk
import joblib
import warnings
import urllib.request
import io
import matplotlib.pyplot as plt
import cv2
import tensorflow as tf
import keras
import sys
import sweetviz as sv
import webbrowser
import gc
import matplotlib.image as mpimg
import shap
import seaborn as sns
from datetime import date, timedelta, datetime
from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import NoSuchElementException
from nltk.corpus import stopwords
from TurkishStemmer import TurkishStemmer
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.model_selection import train_test_split
from sklearn.naive_bayes import GaussianNB
from sklearn.linear_model import LogisticRegression, LinearRegression, SGDRegressor, ElasticNet, BayesianRidge
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.svm import SVC, LinearSVC, SVR
from sklearn.kernel_ridge import KernelRidge
from xgboost.sklearn import XGBRegressor
from lightgbm import LGBMRegressor
#from catboost import CatBoostRegressor
from sklearn.tree import DecisionTreeClassifier
from sklearn.cluster import KMeans
from sklearn import mixture
from sklearn.metrics import accuracy_score, f1_score, precision_score, mean_squared_error, r2_score
from sklearn.model_selection import GridSearchCV
from PIL import Image
from keras.utils import np_utils
from keras.layers.core import Dense, Activation, Dropout, Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D
from keras.optimizers import SGD, RMSprop
from keras.utils import np_utils
from sklearn import metrics
from sklearn.utils import shuffle
from wordcloud import WordCloud
from zemberek import TurkishSentenceNormalizer, TurkishMorphology
from threading import Timer
from scipy import stats
#from IPython.display import display, Image
nltk.download('stopwords', quiet=True)
stemmer = TurkishStemmer()
conn_string = 'host={pghost} port={pgport} dbname={pgdatabase} user={pguser} password={pgpassword}'.format(pgdatabase=db_name,pguser=db_username,pgpassword=db_password,pghost=db_host,pgport=db_port)
conn=psycopg2.connect(conn_string)
cur=conn.cursor()
options = Options()
options.binary_location = chrome_path
warnings.filterwarnings('ignore')
def check_if_table_exists(schema,table):
cur.execute("select exists(select * from information_schema.tables where table_schema='{schema}' AND table_name='{table}')".format(schema=schema, table=table))
return cur.fetchone()[0]
def check_if_index_exists(index):
cur.execute("SELECT EXISTS(SELECT * FROM PG_CLASS WHERE relname = '{index}')".format(index=index))
return cur.fetchone()[0]
def check_if_file_exists(filename):
return os.path.isfile(filename)
def execute_mogrify(conn, df, schema, table):
tuples = [tuple(x) for x in df.to_numpy()]
cols = '"'+'","'.join(list(df.columns))+'"'
cursor = conn.cursor()
try:
for tup in tuples:
query = """INSERT INTO "{schema}"."{table}"({cols}) VALUES ({values}) ON CONFLICT DO NOTHING""".format(schema=schema,table=table, cols=cols, values=",".join(map(str,tup)))
cursor.execute(query)
conn.commit()
except (Exception, psycopg2.DatabaseError) as error:
print("Error: %s" % error)
conn.rollback()
cursor.close()
return 1
cursor.close()
def df_column_conversation(df, column_name, type):
if(type == 'timestamp'):
df[column_name] = df[column_name].apply(lambda x: f"'{x}'::timestamp")
if(type == 'text'):
df[column_name] = df[column_name].str.replace("'","").apply(lambda x: f"'{x}'")
if(type == 'date'):
df[column_name] = df[column_name].apply(lambda x: f"'{x}'::date")
if(type == 'numeric'):
df[column_name] = df[column_name].apply(str).str.replace(',','.')
if(type == 'integer'):
df[column_name] = df[column_name].apply(str).str.replace(',','.').apply(float).astype('Int64').apply(str)
def clean_text(text):
text = text.lower()
text = re.sub(r"[,.\"\'!@#$%^&*(){}?/;`~:<>+=-\\]", "", text)
return text
def webpage_scroll_down(driver, scroll_speed):
total_height = int(driver.execute_script("return document.body.scrollHeight"))
for i in range(1, total_height, scroll_speed):
driver.execute_script("window.scrollTo(0, {});".format(i))
def byte_image_to_numpy_array(byte_image, image_size):
return cv2.resize(np.array(Image.open(io.BytesIO(byte_image))), (image_size, image_size))
def analyze_data(outputHTML,upToDateCheck,schema,table):
if not((check_if_file_exists(outputHTML) and upToDateCheck)):
sql_command = """
SELECT *
FROM "{schema}"."{table}";
""".format(schema=schema, table=table)
df = pd.read_sql(sql_command,conn)
eda = sv.analyze(df)
eda.show_html(outputHTML, open_browser=False)
del df
gc.collect()
#eda_sales.show_notebook()
webbrowser.open(outputHTML,new=2)
print('You can see the exploratory data analysis visualizations in your browser.')
def analyze_dataframe(outputHTML,upToDateCheck,df):
if not((check_if_file_exists(outputHTML) and upToDateCheck)):
eda = sv.analyze(df)
eda.show_html(outputHTML, open_browser=False)
del df
gc.collect()
#eda_sales.show_notebook()
webbrowser.open(outputHTML,new=2)
print('You can see the exploratory data analysis visualizations in your browser.')
def feature_importance_plot(label, model_name, feature_columns, feature_importance):
plt.title('{label} {model_name} Feature Importance'.format(label=label.title(), model_name=model_name))
plt.barh(feature_columns, model.feature_importances_)
plt.tight_layout()
plt.autoscale()
plt.savefig('images/{label}.png'.format(label=label))
plt.close()
def merge_images(image_paths, orientation='horizontal', background_color='WHITE'):
if(type(image_paths[0])!=Image.Image):
images = [Image.open(x) for x in image_paths]
else:
images = image_paths
min_shape = sorted([(np.sum(i.size), i.size) for i in images])[0][1]
if(orientation=='horizontal'):
images_combined = np.hstack((np.asarray(i.resize(min_shape)) for i in images))
if(orientation=='vertical'):
images_combined = np.vstack((np.asarray(i.resize(min_shape)) for i in images))
merged_im = Image.fromarray(images_combined)
merged_image = merged_im
if(background_color!=''):
merged_image = Image.new("RGBA", merged_im.size, background_color)
merged_image.paste(merged_im, (0, 0), merged_im)
return merged_image
def turkish_to_english_characters(text):
tr_en_chars = {'ı':'i', 'ğ':'g', 'İ':'I', 'Ğ':'G', 'ç':'c', 'Ç':'C', 'ş':'s', 'Ş':'S', 'ö':'o', 'Ö':'O', 'ü':'u', 'Ü':'U'}
for tr, en in tr_en_chars.items():
text = text.replace(tr, en)
return text
morphology = TurkishMorphology.create_with_defaults()
normalizer = TurkishSentenceNormalizer(morphology)
def normalize_text(text, timeout=10):
try:
Timer(timeout, sys.exit()).start()
return normalizer.normalize(text)
except:
return text
def stem_text(text):
words = text.split()
stemmed_words = ([stemmer.stem(word) for word in words])
return ' '.join(stemmed_words)
def read_txt_file(location, encoding, column_names):
with open(location, encoding=encoding) as file:
lines = file.readlines()
lines = [line.rstrip() for line in lines]
return pd.DataFrame(lines, columns=column_names)
positive_words_df = read_txt_file('positive_words_tr.txt','utf-8',['WORD'])
negative_words_df = read_txt_file('negative_words_tr.txt','utf-8',['WORD'])
sentiment_words_list = pd.concat([positive_words_df, negative_words_df])['WORD'].values.tolist()
def sentiment_text(text):
sentiment_value = 0
try:
text = text.lower()
word_list = text.split()
for word in word_list:
if word not in(sentiment_words_list):
word_list.remove(word)
for word in word_list:
if word in positive_words_df["WORD"].values:
sentiment_value+= 1
text = text.replace(word, "")
if word in negative_words_df["WORD"].values:
sentiment_value-= 1
text = text.replace(word, "")
finally:
return sentiment_value
def kstest_result(df, column_name, distribution='norm'):
df_name = [x for x in globals() if globals()[x] is df][0]
kstest = stats.kstest(df[column_name][pd.notnull(df[column_name])].values, distribution)
return "Kolmogorov-Smirnov test in {df_name} for {column_name} statistic: {statistic_value:0.4f}, p value: {p_value:0.4f}".format(df_name=df_name, column_name=column_name.capitalize(), statistic_value=kstest[0], p_value=kstest[1])
def distribution_plot(df, column_name):
df_name = [x for x in globals() if globals()[x] is df][0]
df = df[pd.notnull(df[column_name])]
f, (ax_box, ax_hist) = plt.subplots(2, sharex=True, gridspec_kw= {"height_ratios": (0.2, 1)})
mean=df[column_name].mean()
median=df[column_name].median()
mode=df[column_name].mode().values[0]
sns.boxplot(data=df, x=column_name, ax=ax_box)
ax_box.axvline(mean, color='r', linestyle='--')
ax_box.axvline(median, color='g', linestyle='-')
ax_box.axvline(mode, color='b', linestyle='-')
sns.histplot(data=df, x=column_name, ax=ax_hist, kde=True)
ax_hist.axvline(mean, color='r', linestyle='--', label="Mean")
ax_hist.axvline(median, color='g', linestyle='-', label="Median")
ax_hist.axvline(mode, color='b', linestyle='-', label="Mode")
ax_hist.legend()
ax_box.set(xlabel='')
plt.savefig('images/distribution_plot_{df_name}_{column_name}.png'.format(df_name=df_name, column_name=column_name))
plt.close()
